{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "db68f9ed-b3d6-4722-92da-d23681ceaab2",
   "metadata": {},
   "source": [
    "# Exercise 1\n",
    "\n",
    "## Unsupervised Learning\n",
    "\n",
    "---\n",
    "\n",
    "## Overview\n",
    "\n",
    "Welcome to this excercise. We are now going to use our new skills to build our first unsupervised learning models. In particular, we are going to use multiple methods to create a number unsupervisoed learning models such kmeans for clustering and PCA to reduce the dimensionality of or data.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3dcee46f-c65f-44e8-a60a-200a7c537219",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "# Manually set the path relative to the py file's location that you want to import\n",
    "func_lib_path = os.path.abspath(os.path.join(os.getcwd(), '../../'))# Add the path to sys.path\n",
    "\n",
    "# Add the path to sys.path\n",
    "sys.path.append(func_lib_path)\n",
    "\n",
    "# Now you can import func_lib\n",
    "import func_lib\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b6d8b3bc-ab25-4239-be50-b862780e10e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/CD13639-Project/func_lib.py:12: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  sp500_tickers = pd.read_html(response.text)[0]['Symbol'].tolist()\n",
      "[*********************100%***********************]  501 of 501 completed\n"
     ]
    }
   ],
   "source": [
    "historical_prices = func_lib.create_hist_prices()\n",
    "list_of_momentums = []\n",
    "total_returns     = func_lib.compute_returns(historical_prices, list_of_momentums)\n",
    "total_returns.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "71ece63c-7978-4b92-8e7b-8b895f4462fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>F_1_d_returns</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ticker</th>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">A</th>\n",
       "      <th>2000-01-03</th>\n",
       "      <td>-0.076389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-01-04</th>\n",
       "      <td>-0.062030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-01-05</th>\n",
       "      <td>-0.038076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-01-06</th>\n",
       "      <td>0.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-01-07</th>\n",
       "      <td>0.060577</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   F_1_d_returns\n",
       "Ticker Date                     \n",
       "A      2000-01-03      -0.076389\n",
       "       2000-01-04      -0.062030\n",
       "       2000-01-05      -0.038076\n",
       "       2000-01-06       0.083333\n",
       "       2000-01-07       0.060577"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_returns.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ac733284-5436-44ca-aff7-e74dbd3172cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ticker</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>0.000518</td>\n",
       "      <td>0.025869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAPL</th>\n",
       "      <td>0.001183</td>\n",
       "      <td>0.024619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABBV</th>\n",
       "      <td>0.000841</td>\n",
       "      <td>0.016624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABNB</th>\n",
       "      <td>0.000630</td>\n",
       "      <td>0.032406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ABT</th>\n",
       "      <td>0.000532</td>\n",
       "      <td>0.015260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            mean       std\n",
       "Ticker                    \n",
       "A       0.000518  0.025869\n",
       "AAPL    0.001183  0.024619\n",
       "ABBV    0.000841  0.016624\n",
       "ABNB    0.000630  0.032406\n",
       "ABT     0.000532  0.015260"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Group the 'total_returns' DataFrame by the 'Ticker' column.\n",
    "# Calculate the average return ('mean') and standard deviation ('std') of the 'F_1_d_returns' for each group.\n",
    "# Use the 'agg()' function with a list of aggregation functions ['mean', 'std'].\n",
    "# Store the result in a new DataFrame named 'ticker_stats'.\n",
    "ticker_stats = total_returns.groupby('Ticker')['F_1_d_returns'].agg(['mean', 'std'])\n",
    "# Display the first few rows of 'ticker_stats' using the 'head()' method.\n",
    "ticker_stats.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9323e340-c9d9-46cf-9745-e6f88d1cabef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the 'fit_transform()' method of the scaler to standardize the 'mean' and 'std' columns from 'ticker_stats'.\n",
    "# Store the standardized data in a new variable named 'ticker_stats_scaled'.\n",
    "scaler = StandardScaler()\n",
    "ticker_stats_scaled = scaler.fit_transform(ticker_stats[['mean', 'std']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5d47ce45-e533-4c72-9a5e-d68daa8c200d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 39016 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize an empty list to store the inertia values for different numbers of clusters.\n",
    "# Define a range of cluster numbers to test, from 1 to 10.\n",
    "sse = []\n",
    "k_range = range(1, 11)\n",
    "\n",
    "# Iterate over the range of cluster numbers.\n",
    "# For each number of clusters (k):\n",
    "# - Initialize a KMeans object with 'k' clusters, a random state of 42, and 10 initializations.\n",
    "# - Fit the KMeans model to the standardized data ('ticker_stats_scaled').\n",
    "# - Append the inertia (sum of squared distances of samples to their closest cluster center) to the 'sse' list.\n",
    "\n",
    "for k in k_range:    \n",
    "    kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)\n",
    "    kmeans.fit(ticker_stats_scaled)\n",
    "    sse.append(kmeans.inertia_)\n",
    "\n",
    "\n",
    "# Plot the elbow method to visualize the inertia for different numbers of clusters.\n",
    "# - Create a figure with a specified size.\n",
    "# - Plot the inertia values against the number of clusters, using markers for each point.\n",
    "# - Add a title, x-axis label, y-axis label, and grid for clarity.\n",
    "# - Display the plot.\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(range(1, 11), sse, marker='o')\n",
    "plt.title('Elbow Method')\n",
    "plt.xlabel('Number of Clusters')\n",
    "plt.ylabel('Inertia')\n",
    "plt.xticks(range(1, 11))\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c6685cef-5e32-4e76-bab9-724a2a64af75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimal number of clusters: 3\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 95900 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use the function 'plot_optimal_cluster_point' from 'func_lib' to determine the optimal number of clusters.\n",
    "# - Pass the 'sse' list (containing the inertia values) and 'k_range' (the range of cluster numbers) as arguments.\n",
    "# - This function will help visualize and identify the \"elbow\" point in the plot, indicating the optimal number of clusters.\n",
    "\n",
    "func_lib.plot_optimal_cluster_point(sse, k_range)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d8782bd8-5b4a-4817-b76f-10417ee486b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>Cluster</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.639675</td>\n",
       "      <td>0.430455</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.230171</td>\n",
       "      <td>0.244291</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.269430</td>\n",
       "      <td>-0.946556</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.323661</td>\n",
       "      <td>1.404073</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.601442</td>\n",
       "      <td>-1.149764</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       mean       std  Cluster\n",
       "0 -0.639675  0.430455        1\n",
       "1  1.230171  0.244291        1\n",
       "2  0.269430 -0.946556        0\n",
       "3 -0.323661  1.404073        1\n",
       "4 -0.601442 -1.149764        0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert the standardized data to a DataFrame if it is not already in that format.\n",
    "# Use 'pd.DataFrame()' to create a DataFrame from 'ticker_stats_scaled', specifying the column names as 'mean' and 'std'.\n",
    "ticker_stats_df = pd.DataFrame(ticker_stats_scaled, columns=['mean', 'std'])\n",
    "\n",
    "# Define the number of clusters for K-Means clustering.\n",
    "# Set the variable 'k' to 3 to specify the number of clusters.\n",
    "k = 3\n",
    "\n",
    "# Perform K-Means clustering.\n",
    "# Initialize a KMeans object with 'n_clusters' set to 'k' and 'random_state' set to 42.\n",
    "# Fit the KMeans model to 'ticker_stats_df' and predict the cluster labels.\n",
    "# Store the predicted cluster labels in the variable 'clusters'.\n",
    "kmeans = KMeans(n_clusters=k, random_state=42)\n",
    "clusters = kmeans.fit_predict(ticker_stats_df)\n",
    "\n",
    "# Add the cluster labels to the DataFrame.\n",
    "# Create a new column named 'Cluster' in 'ticker_stats_df' and assign the cluster labels to this column.\n",
    "# Display the first few rows of 'ticker_stats_df' to check the added cluster labels.\n",
    "ticker_stats_df['Cluster'] = clusters\n",
    "ticker_stats_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "253ff2eb-646d-4f10-8814-cbd15293a151",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 48764 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create a new figure with a specified size for the plot.\n",
    "# Iterate over each cluster (from 0 to k-1).\n",
    "# Use 'plt.scatter()' to plot the 'std' (standard deviation) vs. 'mean' (average return) for each cluster.\n",
    "# Filter the 'ticker_stats' DataFrame using the cluster labels from 'ticker_stats_scaled'.\n",
    "# Use different colors and labels for each cluster.\n",
    "# Add a title, x-axis label, y-axis label, and a legend to the plot.\n",
    "# Display the grid for better readability.\n",
    "# Finally, use 'plt.show()' to display the plot.\n",
    "\n",
    "plt.figure(figsize=(10, 6))\n",
    "for cluster in range(k):\n",
    "    plt.scatter(ticker_stats_df[ticker_stats_df['Cluster'] == cluster]['std'],\n",
    "                ticker_stats_df[ticker_stats_df['Cluster'] == cluster]['mean'],\n",
    "                label=f'Cluster {cluster}', s=1)\n",
    "plt.title('K-Means Clustering of Tickers based on Average Return and Standard Deviation')\n",
    "plt.xlabel('Standard Deviation')\n",
    "plt.ylabel('Average Return')\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
